# -*- coding: utf-8 -*-
import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from scrapy.http import Request as sreq
from girl.items import city
from datetime import *
class NewdiarycitySpider(CrawlSpider):
    name = "newdiarycity"
    allowed_domains = ["cityheaven.net"]
    start_urls = [
        'https://www.cityheaven.net/hokkaido/diarytop/diarynewlist/typ609/',
         # https://www.cityheaven.net/hokkaido/diarytop/diarynewlist/
    ]
    rules = (
        Rule(LinkExtractor(deny=('.*/---'),
        restrict_xpaths=("//div[@class='shop_nav_list']//a")),
        follow=True,callback="parse1"),
        # Rule(LinkExtractor(deny=('.*/---'),allow=('movie')
        # restrict_xpaths=("//url[@class='diarynewlist_contents clearfix masonry']//a")),
        # callback="parse0"),
    )

    tmp = {'girl.newdatapipelines.NewDataPipeline': 300}
    custom_settings = {
        'ITEM_PIPELINES':tmp
    }
    # def __init__(self, *a, **kw):
    #     super(NewdiarycitySpider, self).__init__(*a, **kw)

    def parse_start_url(self, response):
        areas = response.css('.popbox li a::attr("href")').extract()
        del areas[0]
        del areas[1]
        urls = ['https://www.cityheaven.net%sdiarytop/diarynewlist/typ609/' % x for x in areas]
        urls = [sreq(x,callback=self.sourceNum) for x in urls]
        return urls

    def sourceNum(self,response):
        sdata = self.crawler.stats.get_value('sdata',0)
        sdata = int(sdata) + int(response.css(".red.c_red::text").extract_first())
        self.crawler.stats.set_value('sdata',sdata)

    def parse1(self, response):
        f = response.css(".diarynewlist_contents li")
        item = city.diaryItem()
        for x in f:
            item['image_urls'] = x.css(".main_img img::attr('data-echo')").extract_first()
            if not item['image_urls']:
                item['image_urls'] = x.css(".main_img img::attr('src')").extract_first()
            item["time"] = x.css(".fz10::text").extract_first()
            item["title"] = x.css(".bold::text").extract_first()
            item["girlUrl"] = x.css(".text-ellips::attr('href')").extract_first()
            item["url"] = x.css(".bold::attr('href')").extract_first()
            item['date'] = str(date.today())
            yield item
